import requests,time
from lxml import etree
from datetime import datetime
from urllib import parse
from tinydb import where
from common import UID,HandleTmpList,parseContentToName,ProductToGroup,sess,ProductNameTuple
SPIDERNAME='期货日报'
def parseQHRBArtPubTime(string):
    #2020-11-22 22:16
    return string+':00'

def parseQHRBArtContent(url):
    r=sess.get(url)
    r.encoding='utf8'
    selector=etree.HTML(r.text)
    try:
        ele=selector.cssselect("div.article-content")[0]
        content=ele.xpath("string(.)").strip().strip('\n')
    except:
        print('网页内容为空',url)
        content=''
    # print(content)
    return content

def getQHRBArticleList(articleCol,BeCrawledUrlList):
    # print(SPIDERNAME)
    # #农产品
    # url1='http://www.qhrb.com.cn/farm/'
    # #金属
    # url2='http://www.qhrb.com.cn/metal/'  
    # #能源化工
    # url3='http://www.qhrb.com.cn/energy/'
    # #实市场告
    # url4='http://www.qhrb.com.cn/comment/scbg/'



    temp_article_ls=[]
    for product in ProductNameTuple:
        url='http://app.qhrb.com.cn/?app=search&controller=index&action=search&wd=%s&advanced=1&type=all&catid=&order=time'%parse.quote(product)
        
        try:
            r=sess.get(url,timeout=20)
        except:
            print('期货日报超时',url)
            continue
        
        r.encoding='utf8'    #不做任何设置的时候，正确
        selector=etree.HTML(r.text)
        
        #之前好几年的内容放在一页，只取5*12条
        eleList=selector.cssselect(".search-list dd")
        
        for ele in  eleList:
            
            # time.sleep(0.5)
            articleUrl=ele.xpath("./div/a/@href")[0]
            #判断是否已经爬取过，如果是，跳出循环
            if len(articleCol.search(where('articleUrl')==articleUrl)):break
            # if articleUrl in BeCrawledUrlList:break
            title=''.join(ele.xpath('./div/a/text()'))
            try:
                publicTime=parseQHRBArtPubTime(ele.xpath('./p/var/text()')[0])
            except:
                print(url,'   ',title,'   找不到时间字符串')

            temp_dict={'tags':['期货日报'],'score':0,'uid':UID()}
            temp_dict['title']=title.strip()
            temp_dict['articleFrom']='期货日报'
            temp_dict['url']=articleUrl.strip()
            temp_dict['publicTime']=publicTime.strip()
            #文章内容
            content=parseQHRBArtContent(articleUrl)
            temp_dict['content']=content
            #定文章所属期货品种,板块
            # n=parseContentToName(title+content)
            
            
            temp_dict['product_name']=product
            print(SPIDERNAME,'   ',title,"    ",product)
            temp_dict['group']=ProductToGroup[product]
            
            
            temp_article_ls.append(temp_dict)

    #注意缩进不要错
    HandleTmpList(temp_article_ls,articleCol,SPIDERNAME)
    # print('mysteel: %d  条'%articleCol.count_documents({'from':'mysteel'}))